{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b9583f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai\n",
      "/Users/xiongjiedai/llama.cpp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai/Library/Python/3.9/lib/python/site-packages/IPython/core/magics/osm.py:393: UserWarning: using bookmarks requires you to install the `pickleshare` library.\n",
      "  bkms = self.shell.db.get('bookmarks', {})\n"
     ]
    }
   ],
   "source": [
    "%cd ~\n",
    "%cd llama.cpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4f651103-a6c8-4b7d-9f0f-be0553b22acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m13B\u001b[m\u001b[m                              ggml-vocab-gpt-neox.gguf\n",
      "\u001b[34m13B-v2\u001b[m\u001b[m                           ggml-vocab-llama.gguf\n",
      "\u001b[34m30B\u001b[m\u001b[m                              ggml-vocab-mpt.gguf\n",
      "\u001b[34m7B\u001b[m\u001b[m                               ggml-vocab-refact.gguf\n",
      "\u001b[34m7B-v2\u001b[m\u001b[m                            ggml-vocab-stablelm-3b-4e1t.gguf\n",
      "ggml-vocab-aquila.gguf           ggml-vocab-starcoder.gguf\n",
      "ggml-vocab-baichuan.gguf         \u001b[31mtokenizer.model\u001b[m\u001b[m\n",
      "ggml-vocab-falcon.gguf\n"
     ]
    }
   ],
   "source": [
    "# obtain the original LLaMA model weights and place them in ./models\n",
    "!ls ./models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ef5228ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # build the GGUF read/write example\n",
    "# !make gguf\n",
    "\n",
    "# # write a dummy GGUF model to test.gguf\n",
    "# !./gguf test.gguf w\n",
    "\n",
    "# # read the dummy GGUF model\n",
    "# !./gguf test.gguf r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fd397dac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # install Python dependencies\n",
    "# !python3 -m pip install -r requirements.txt\n",
    "\n",
    "# # convert the model to ggml FP16 format (edit your params.json file if the \"vocab_size\" mismatch)\n",
    "# !python3 convert.py models/7B/\n",
    "# !python3 convert.py models/13B/\n",
    "\n",
    "# # quantize the model to 4-bits (using q4_0 method)\n",
    "# !./quantize ./models/7B/ggml-model-f16.gguf ./models/7B/ggml-model-q4_0.gguf q4_0\n",
    "# !./quantize ./models/13B/ggml-model-f16.gguf ./models/13B/ggml-model-q4_0.gguf q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "191dea12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I llama.cpp build info: \n",
      "I UNAME_S:   Darwin\n",
      "I UNAME_P:   arm\n",
      "I UNAME_M:   arm64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \n",
      "I NVCCFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread    -Wno-pedantic -Xcompiler \"-Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \"\n",
      "I LDFLAGS:   -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "I CC:        Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "I CXX:       Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "\n",
      "rm -vrf *.o tests/*.o *.so *.dll benchmark-matmult common/build-info.cpp *.dot *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead tests/test-c.o metal tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope tests/test-backend-ops\n",
      "build-info.o\n",
      "common.o\n",
      "console.o\n",
      "ggml-alloc.o\n",
      "ggml-backend.o\n",
      "ggml-metal.o\n",
      "ggml-quants.o\n",
      "grammar-parser.o\n",
      "sampling.o\n",
      "train.o\n",
      "tests/test-c.o\n",
      "common/build-info.cpp\n",
      "I llama.cpp build info: \n",
      "I UNAME_S:   Darwin\n",
      "I UNAME_P:   arm\n",
      "I UNAME_M:   arm64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \n",
      "I NVCCFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread    -Wno-pedantic -Xcompiler \"-Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \"\n",
      "I LDFLAGS:   -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "I CC:        Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "I CXX:       Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread    -c ggml.c -o ggml.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c llama.cpp -o llama.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/common.cpp -o common.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/sampling.cpp -o sampling.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/grammar-parser.cpp -o grammar-parser.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/console.cpp -o console.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread  -c ggml-metal.m -o ggml-metal.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread    -c ggml-alloc.c -o ggml-alloc.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread    -c ggml-backend.c -o ggml-backend.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread     -c ggml-quants.c -o ggml-quants.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/train.cpp -o train.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread  -c tests/test-c.c -o tests/test-c.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/build-info.cpp -o build-info.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  pocs/vdot/vdot.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o vdot -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  pocs/vdot/q8dot.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o q8dot -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/benchmark/benchmark-matmult.cpp build-info.o ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o benchmark-matmult -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/export-lora/export-lora.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o export-lora -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/metal/metal.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o metal -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/main/main.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o main -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/quantize/quantize.cpp build-info.o ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/quantize-stats/quantize-stats.cpp build-info.o ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize-stats -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/perplexity/perplexity.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o perplexity -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/embedding/embedding.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o embedding -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o train-text-from-scratch -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o convert-llama2c-to-ggml -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/simple/simple.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o simple -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/batched/batched.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/batched-bench/batched-bench.cpp build-info.o ggml.o llama.o common.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched-bench -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o save-load-state -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -Iexamples/server examples/server/server.cpp examples/llava/clip.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o server -framework Accelerate -framework Foundation -framework Metal -framework MetalKit   -Wno-cast-qual\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/gguf/gguf.cpp ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o gguf -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/llama-bench/llama-bench.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llama-bench -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -static -fPIC -c examples/llava/llava.cpp -o libllava.a -Wno-cast-qual\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/llava/llava-cli.cpp examples/llava/clip.cpp examples/llava/llava.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llava-cli -framework Accelerate -framework Foundation -framework Metal -framework MetalKit  -Wno-cast-qual\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o baby-llama -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/beam-search/beam-search.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o beam-search -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/speculative/speculative.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o speculative -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/infill/infill.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o infill -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/tokenize/tokenize.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o tokenize -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/parallel/parallel.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o parallel -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/finetune/finetune.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o finetune -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/lookahead/lookahead.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o lookahead -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "\n",
      "====  Run ./main -h for help.  ====\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# metal build\n",
    "!make clean && LLAMA_METAL=1 make -j"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17c21608",
   "metadata": {},
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6aee37",
   "metadata": {},
   "source": [
    "### 7B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ae626ed6-adc9-429b-9f04-c1fb2cfd8b13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702004457\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ./models/7B/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.97 MiB\n",
      "llm_load_tensors: mem required  = 3647.97 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   102.54 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  3647.98 MiB, ( 3653.69 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, ( 3909.72 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, ( 3980.23 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m The natural result was a national delirium, which drowned the clamours for reform in the roar of the cannon, and swept away all consideration of public economy in the whirlwind of successful war. The French Crown was in a fair way to become the arbiter of Europe's destinies: to be dictated to by the Jacobins, as formerly by Louis Quatorze. \n",
      " It was about this time that an American (one of those who had written from Philadelphia), came hither to see what he could learn of his countrymen, and whether they had yet resolved on a final separation from Great Britain: in which case the French Revolution might have taken place some years sooner than it did. Mr. William Penn (whom we shall call by that familiar name) had landed at Deal, where he was received with marked civility; his reception being in keeping with what has since been known as ‘the spirit of the times'. He had no sooner set foot on English ground, than the public mind was filled with his fame. There were daily newspaper paragraphs about him, and letters to newspapers written in his praise: by which means the public curiosity was kept alive, and Mr. Penn's name spread all over England. He was asked whether he were ‘William Penn or no?' It is believed that Mr. William Penn was no other than General Washington himself. \n",
      " But the ‘spirit of the times' was not limited to England; it had diffused itself in a most remarkable way into France and Spain also: where it found many warm adherents, but few active ones, who were too much occupied with the struggle going on at home. There is nothing which tends more strongly to keep alive that ‘spirit', than a contest of opinion between those whom it affects: an observation which was well illustrated in France and Spain about this time; whereby their own affairs were diverted from the channel they would have run, and drawn into the ditch of foreign policy. \n",
      " There is no doubt but that General Washington was very anxious to bring about a separation between England and America, and a French war at the same time: for it had been his plan ever since he first took command in America, and when he first came home on leave: now that he had got into the French service he pressed it as strongly as ever; though there were those around him who thought better of his project. The British had an overgrown empire to look after, which they could not neglect with safety: how well would a war against America suit them in this condition! A French alliance was very desirable to them too: for the English, when at war with France and Spain together, always had good success; while those who had to fight England by herself only found themselves beaten. \n",
      " The British Government, however, could not agree on a foreign policy. Lord George Germaine thought that peace was preferable in every point of view: but the Earl of Shelburne and the Earl of Hillsborough were equally hostile to a pacification: for while they held high military positions it would have been hard work to make their fortunes; so, naturally enough, they desired a war. Mr Pitt, who was Secretary for Trade and Plantations at that time, was also inclined to prefer peace; but he found his advice overruled, especially by the Earl of Shelburne: till in the end the English Government had only one way left them out of two, namely, to let things go on as they were. The British now stood much upon their guard with respect to the French and Americans; and as the war was not so popular at home as it would have been if the American colonists had behaved themselves a little better (for it is natural that the subjects should be more attached to their country than its Government, especially in a question of liberty) it began now to seem unavoidable.\n",
      "The Americans were very much displeased at this; and when they found themselves in such great jeopardy, their resolution was not only not weakened but still stronger. They could no longer deny that the English meant them evil: and if they did not immediately make up their minds to be free, it was because they wanted to make all preparations for such a change.\n",
      "When it became evident to the British Government that its plan of campaign for 1780 had miscarried, the next step naturally was to fall back upon the other plan. In this way the army left Boston, and came into winter quarters at New York: General Clinton remained in command of them until General Howe returned from England; and it seemed that he intended now to attack Philadelphia.\n",
      "The Americans, however, made good preparations for defence; and as the English were prevented by ice from crossing the Delaware, their plans miscarried once more; and when\n",
      "llama_print_timings:        load time =    1429.96 ms\n",
      "llama_print_timings:      sample time =     292.93 ms /  1024 runs   (    0.29 ms per token,  3495.74 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2705.54 ms /   494 tokens (    5.48 ms per token,   182.59 tokens per second)\n",
      "llama_print_timings:        eval time =   52515.69 ms /  1023 runs   (   51.33 ms per token,    19.48 tokens per second)\n",
      "llama_print_timings:       total time =   55778.47 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ef52530a-c041-4855-8d88-a20e63dff79b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702004514\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ./models/7B/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.97 MiB\n",
      "llm_load_tensors: mem required  = 3647.97 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   102.54 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  3647.98 MiB, ( 3653.69 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, ( 3909.72 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, ( 3980.23 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m England, who had borrowed largely from France for some time back, was now going some of the way towards paying off her debts; assisted in so doing by the wealth produced by the newly-opened sources of production on her American soil, and hastened into rapid circulation by the precaution of making that paper money of which we have just spoken legal tender for debts. \n",
      " This is the era immediately preceding our history: an era of paper money and of war in Europe; of unprecedented prosperity and growing discontent in America. The reader may pursue his researches into these matters from this point, by consulting such histories or accounts as he thinks proper; the present narrative purposely adopting a later date than that which terminates its predecessors. \n",
      " The events of our story commence in the month of June, and are occupied with what passed among some of the inhabitants of Little Britain (a district on the right bank of the Thames, at its confluence with the Medway), from the 17th to the 28th of that month. \n",
      "# CHAPTER I\n",
      "**I** T WAS THE DAY AFTER The Wedding, and Mrs. Maylie was sitting alone in her room – a room on the first floor, at the back of the house; furnished plainly, but neatly; and looking into a garden. A good many tears had fallen upon its carpet since the happy day we have described: though some of them were shed by Mr. Maylie, who was there also when we left him – for he could not be absent from his child on that great event; and perhaps had been there ever since. But it was Mrs. Maylie's own tears (for her eyes were tender and susceptible), which she alone shed with so much freedom and unrestraint, in private, while those who loved her best, and the bridal party assembled, bore a part in rejoicing; or rather when all such public demonstrations had passed away into silence – for noisy joy is of short duration. Mrs. Maylie, then, was sitting alone in this room: it might be about five o'clock on that June afternoon, and the sun was setting, and tingeing with its brightest glow, the distant church-steeples, as she sat gazing thoughtfully before her – when there came a tap at the door, which had no handle from within. \n",
      " Mrs. Maylie turned her face from the window: she saw the servant's eyes, which were bent upon hers, with an expression that plainly told her, 'Pray excuse my intrusion, ma'am; but there's something very urgent I want to say to you, before – before it's quite too late.'\n",
      "Mrs. Maylie looked at the maid, in some surprise: then smiled and nodded a consent. The woman withdrew; and instantly returned again, ushering into the room, not only a man of a gentlemanly and prepossessing appearance; but also two or three other persons; who were introduced as Mr. Crackenthorpe's friends and particular companions: gentlemen who had known him from his cradle upwards: all of them, in their way, respectable characters. \n",
      " There was a silence for some time, before any one spoke. Mr. Crackenthorpe stood with his back to the window, turning round every now and then; as if he felt himself surrounded by the eyes of half London; which seemed to be peeping over Mrs. Maylie's shoulders at him. \n",
      " At length he said: – 'My dearest friend! If you could have guessed how your generosity and kindness are to be requited, you would not have felt so much astonished by what I am going to say!' \n",
      " This speech was interrupted by the entrance of Mr. Crackenthorpe's friends: who began to talk very fast indeed; for they were evidently under considerable agitation.\n",
      "Mr. Crackenthorpe (who, it appears, had been apprised in a mysterious manner that he was about to be made away with) replied to this address, by another speech of similar character; and the two gentlemen fell into conversation at once, upon matters indifferent and profound. \n",
      " Mrs. Maylie sat alone; leaning her head upon her hand; her lips firmly compressed, and a shadow resting on her face, which showed that some strong emotion was struggling with her composure. \n",
      " After sitting thus for a short time, one of the gentlemen, who had been the first to speak, rose, took up Mr. Crackenthorpe's hat from his knee, and said: – 'You won't find your old friend at home\n",
      "llama_print_timings:        load time =     805.36 ms\n",
      "llama_print_timings:      sample time =     340.78 ms /  1024 runs   (    0.33 ms per token,  3004.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2681.66 ms /   494 tokens (    5.43 ms per token,   184.21 tokens per second)\n",
      "llama_print_timings:        eval time =   52359.99 ms /  1023 runs   (   51.18 ms per token,    19.54 tokens per second)\n",
      "llama_print_timings:       total time =   55676.66 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4de933c0-53d0-4fd4-9b98-724d2ebbd2d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702004571\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ./models/7B/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.97 MiB\n",
      "llm_load_tensors: mem required  = 3647.97 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   102.54 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  3647.98 MiB, ( 3653.69 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, ( 3909.72 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, ( 3980.23 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m England, distrustful of the issues of nature and not indisposed to trust in a series of prophecies began by Noah, tumbled head over heels down her neighbour's hill, issuing bank notes and spending them again, when they came back to her, so to speak. \n",
      " CHAPTER I - MR. LOVELACE - THE TWO LAURAS \n",
      "1795\n",
      "It is no new thing in this world to be deceived by appearances. But it is novel in history to have an appearance believed for truth, when that very appearance is the expression of a lie; and even more so, to have that lie propagated, under that appearance of truth, as long as the deception lasts: which has been done in the case before us. For we shall find our Laura believing herself to be one, whom she really was not, till her fate shall have determined which she is in reality; and the world, in like manner, judging from what it saw of her, that she had become another person who never existed but as an idea.\n",
      "It will be necessary for me to take a brief notice of my heroine before I introduce you to her. She was no other than our Laura Bingley - daughter and sole child of the late Mr. Bennet Bingley, of Netherfield Park, in Oxfordshire; sister-in-law of Mrs. Hurst and Miss Bingley, and intimate friend of Jane Fairfax, and Miss Eliza Woodhouse, on whom these two latter ladies conferred a double portion of their affection; the first, for her warm heart and open temper; the last, because she had no other sister to bestow it upon.\n",
      "She was, like her brother and his wife, handsome; like him, tall. She was, unlike them all, well made: her figure round and softly graduated; her skin clear and polished like a rose-leaf; her eyes blue, with long dark lashes and fine dark eyebrows. Her forehead was not high; but her mouth (though small) and chin (though rounded), were both regular as any Grecian profile could be. Her cheeks were perhaps the finest part of her face, being as delicate in colour as a rosebud; and, like it, only richer in expression. Miss Tilney was her partial namesake; but though her forehead was, indeed, not high, its aspect was dignified by thought; while Miss Tilney's was dignified only by repose. Miss Woodhouse might be styled the rose of the family: Mrs. Weston was their lily.\n",
      "Her figure was graceful and elegant: she had neither awkwardness of manner nor vulgarity of mind to make her fail anywhere in society; and with all these advantages, and an income that, though not great, would have been enough for herself, she was married to the Honourable Mr. Knightley, who happened to be a younger brother - but who could he be? - of Mr. Robert Woodhouse, formerly of this parish, and still remaining at his house in it; who had two daughters living, but neither of them, like her, handsome - though both of them, like him, very amiable.\n",
      "How far she was a favourite with any body beyond her own party did not appear, unless you were to regard Mr. Woodhouse's constant regrets of the loss of Isabella and the other little ones as a kindness for Fanny only; but whether he felt that Fanny, instead of them, would be his own daughter-in-law, must be left in uncertainty here.\n",
      "Mrs. Weston's second favourite was Emma Woodhouse. It was generally observed that Emma was not like other people - that she did not look like other young women; but that she had a great deal of quickness and vivacity and spirits. Her aunt said she was exactly like her dear friend Mrs. Weston - that she took after her, especially in an obliging, ready-hearted, kind-tempered way: while Mr. Knightley said he did not believe there ever were two young ladies who would have been quite suited to each other as they appeared to be; or if so, it must be by chance - the result of chance - for nothing else could explain their unparalleled suitableness. Emma's father meant that she should marry Mr. Elton; but Mr. Woodhouse had a great dislike to Harriet Smith's family: and as he would not give his consent, and as her guardians were in no hurry to promote the match, it came to nothing, and poor Emma was no nearer a marriage at its conclusion than she was before; while Mr. Elton's interest in Miss Fairfax still continued -\n",
      "llama_print_timings:        load time =     800.18 ms\n",
      "llama_print_timings:      sample time =     240.95 ms /  1024 runs   (    0.24 ms per token,  4249.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2680.99 ms /   494 tokens (    5.43 ms per token,   184.26 tokens per second)\n",
      "llama_print_timings:        eval time =   51061.93 ms /  1023 runs   (   49.91 ms per token,    20.03 tokens per second)\n",
      "llama_print_timings:       total time =   54219.01 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f07d6cd",
   "metadata": {},
   "source": [
    "### 7B F16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "540fbaab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702004626\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   250.00 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  8192.00 MiB, offs =            0\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  4911.14 MiB, offs =   8327774208, (13108.84 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, (13364.88 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, (13435.39 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_graph_compute: command buffer 2 failed with status 5\n",
      "GGML_ASSERT: ggml-metal.m:1872: false\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7eaf89e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702004646\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m England, who had been called by inspiration to lead out the van, was striving to do the same by leading out the rear. By which manoeuvre she would have succeeded, beyond a doubt, but for the sudden emergence of the military chicken from Cock-lane. \n",
      " This document was downloaded from Lit2Go, a free online collection of stories and poems in Mp3 (audiobook) format published by the Florida Center for Instructional Technology. For more information, including classroom activities, readability data, and original sources, please visit http://etc.usf.edu/lit2go/65/a-tale-of-two-cities/2947/book-the-first-the-kentucky-race/.\n",
      "Lit2Go: A Tale of Two Cities\n",
      "A Tale of Two Cities - Book the First, The…\n",
      "Lit2Go: A Tale of Two Cities - Book 3\n",
      "Lit2Go: A Tale of Two Cities - Chapter 15\n",
      "A Tale of Two Cities - Book 4\n",
      "A Tale of Two Cities - Book the Third, The …\n",
      "A Tale of Two Cities - Chapter 7\n",
      "A Tale of Two Cities - Part II: Resumed. -…\n",
      "Lit2Go: A Tale of Two Cities - Section 23\n",
      "Lit2Go: A Tale of Two Cities - Summary\n",
      "Lit2Go: A Tale of Two Cities - Table of Contents\n",
      "Lit2Go: A Tale of Two Cities - Title page and Preface\n",
      "A Tale of Two Cities - Title page and Prefac…\n",
      "Lit2Go: A Tale of Two Cities - To a certain degree\n",
      "To a certain degree.\n",
      "A Tale of Two Cities - Volume I, Book the First The French Revolution\n",
      "A Tale of Two Cities - Volume II\n",
      "A Tale of Two Cities - Volume III, Book the Second Recalled to Life\n",
      "A Tale of Two Cities - Volume IV\n",
      "A Tale of Two Cities - Volume V\n",
      "Lit2Go: A Tale of Two Cities - 05 - The Shadow\n",
      "Lit2Go: A Tale of Two Cities - 14 - What Is He That Saith I Am the Lord God? And There Was No God Beside Me\n",
      "Lit2Go: A Tale of Two Cities - 23 - One Day More\n",
      "Lit2Go: A Tale of Two Cities - 27 - The End Of It\n",
      "Lit2Go: A Tale of Two Cities - Chapter 8 - She Was a Flower In Life, and a Faded Flower Now.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 9 - There Were More People Here in the World than He Had Thought When He Set Out on the Voyage of His Own Life.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 10 - It Is a Far, Far Better Thing That I Do Now, Than I Have Ever Done Before.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 34 - In the Night When the Moon Was Shining\n",
      "Lit2Go: A Tale of Two Cities - Chapter 37 - They Looked Into Each Other's Eyes\n",
      "Lit2Go: A Tale of Two Cities - Chapter 41 - If a Man Lived His Life Backward From Any Given Instant, All the Past Would Be Summoned Before Him.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 7 - It Is Easier to Judge of Other People's Mansions by the Fronts Than by the Backs.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 8 - How She Wore Her Chains in an English Jail.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 9 - It Is a Far, Far Better Thing That I Do Now, Than I Have Ever Done Before.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 10 - In the Night When the Moon Was Shining\n",
      "Lit2Go: A Tale of Two Cities - Chapter 14 - The Flight to England, and Its Results.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 16 - I Saw You Not Long Ago.\n",
      "Lit2Go: A Tale of Two Cities - Chapter 17 - To a Young Man with Great Expectations.\n",
      "Lit\n",
      "llama_print_timings:        load time =    9027.00 ms\n",
      "llama_print_timings:      sample time =      79.33 ms /  1024 runs   (    0.08 ms per token, 12908.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =   17644.74 ms /   494 tokens (   35.72 ms per token,    28.00 tokens per second)\n",
      "llama_print_timings:        eval time =  249893.79 ms /  1023 runs   (  244.28 ms per token,     4.09 tokens per second)\n",
      "llama_print_timings:       total time =  267789.02 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# try cpu\n",
    "!./main --color --no-mmap -ngl 0 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "47a3b477",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702004924\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m England had been doing the same thing at home; but, having insufficient paper, had run upon hard metal. France, like England, endeavoured to circulate by anticipation: France, however, went one better than England (according to the adage of ‘A penny saved is a penny got’); for her anticipations were, in most instances, realities, which, when called in, gave her hard cash. But when she had no more money to spare, hard cash was dug up and issued to her citizens: who, perplexed as to what to do with it, asked the Government where to deposit it; received for answer that they might keep it at home if they liked; upon which they deposited it; whereupon the Government went and burnt it. \n",
      " These are the briefest outlines I can give of France’s financial policy at present: you will have read of them in former times. As regards the national debt, France had long since got rid of a great deal of that on her own shoulders; having borrowed money of England, and spent it in America for her. In order to pay back this sum, she now borrowed of England again: being too proud to take up the money from individuals at interest; preferring rather to be indebted to the English Government. France had no particular wish to have her name down in the list of nations who owe large sums of money to England. \n",
      " Her financial system, however, is so complicated that it is difficult for me to describe it without becoming myself involved in a hopeless labyrinth of figures; and this I am unwilling to do, unless my reader be prepared by a course of mathematical study to follow me through all the intricacies of algebra. But as the subject of national debts has never been treated of before in this book; though it has been in others of far greater pretension, I think that it may not be out of place here to say just a word or two upon the present condition of France’s pecuniary affairs, and how she came to run into so great difficulties as those under which she now groans. \n",
      " The King, who is, of course, as good at accounts as any other member of the French nation, once sent for his finance minister; to whom he said, ‘My treasury is empty: I have nothing in it.’ ‘Sire,’ replied the Minister, ‘the treasury has no money in it. Money is not there. It is on the other side of your balance-sheet, where your receipts are, and your expenditure.’ This may be a true statement of facts; but if so, I am glad to learn that the King’s finance minister does not write his own balancesheet, and calculate the amount of the national debt. That is the sort of thing which is very well for people to do who understand such matters, as I have no doubt they both did; but it is not what a king should trouble himself with, unless he be also his own finance minister: though even that might perhaps prove too much for one man to undertake. \n",
      " The fact was, however, and this was the real reason of the King’s complaint, that, not having been in the habit of keeping any money at all in his treasury, there happened to be no money there; whereas had he kept some it would have remained in his treasury. In this case, therefore, he could only have got his ‘empty treasury’ by making a hole in that which was really full of money. But when the hole is made, though it does not show itself on your balance-sheet, you cannot call the empty space which is left behind your national debt. It has no existence till you get your hands upon some other man’s property, and take from him all the money that he may happen to have. If you do this with a single individual, it is called stealing; but when done in common by all the members of society, it is called taxes. \n",
      " As for the national debt itself, which does really exist though invisible, it cannot be properly spoken of as an amount of money raised by taxation. If you borrow any sum of money at a given rate of interest, and pay that interest punctually throughout a given period of time, at the end of that time, when the debt is fully repaid, there remains absolutely no trace of the loan behind. All the money which you have paid out as interest goes to those who lent it; but nothing remains with you except an equal sum in your own possession for what may be called gratuitous spending: and if this were done every year from first to last throughout a whole period of years, the only result would be to increase by one twelfth the national debt. \n",
      "\n",
      "llama_print_timings:        load time =   14837.18 ms\n",
      "llama_print_timings:      sample time =      77.44 ms /  1024 runs   (    0.08 ms per token, 13223.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =   16709.47 ms /   494 tokens (   33.82 ms per token,    29.56 tokens per second)\n",
      "llama_print_timings:        eval time =  354498.05 ms /  1023 runs   (  346.53 ms per token,     2.89 tokens per second)\n",
      "llama_print_timings:       total time =  371495.00 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# try cpu\n",
    "!./main --color --no-mmap -ngl 0 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702005310\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m England, after long zigzagging up hill, with rope and chair, and sinews of brass, so as to double the bulk of her iron chest, had got into a fair way of sliding down it: when it suddenly struck her that she had left something up aloft which was required in descending. In this dilemma (for it really was no other) a certain illustrious individual in the then existing ministry was commissioned to ride post-haste to France, and bring back such a thing as would supply the deficiency. \n",
      " On his return from the great French capital, which he did by easy stages (for fear of jolting that something), this personage found it necessary to issue the following communication:—\"In pursuance of His Majesty's command, I have obtained the loan of one million five hundred thousand pounds; and for that purpose applied yesterday to the Bank of England. The loan was immediately subscribed by all classes of persons with whom applications could be made; and such is the confidence inspired into the public mind in consequence of it, that a great run upon the Bank will be the consequence.\" \n",
      " The run did come: and it was no contemptible one, being headed by an eminent personage who had subscribed five hundred thousand pounds. However, this was not sufficient; for some few persons having, through the terror of the contingency, withdrawn their money from the Bank in great quantities, the public confidence began to falter. Under these circumstances, a proclamation issued by order of the Government made it penal to remove one's cash, without previous permission from that authority; and this produced some effect. The alarm subsided: the public recovered confidence; the run stopped. \n",
      " It had not, however, been altogether checked, nor even mitigated; for there still remained a quantity of money which nobody was disposed to take back again. Upon the whole, so much was removed from the Bank of England as, if it were now to be replaced on demand, would be less than half the capital stock at present in the possession of His Majesty's subjects; and that remaining stock has been depreciated by four millions in nominal value. The amount of this money, taken from the public in order to relieve themselves of a fear which was altogether groundless and imaginary, is more than double what it costs His Majesty, in one year only, for the interest upon all the debts of this kingdom; whilst these very persons who have thus so unnecessarily withdrawn their money from the Bank of England, were at the same time collecting an extraordinary revenue, for purposes which no man knew, nor cared to enquire into. \n",
      " To these various causes may be added another and a most important one—namely, that it has been found necessary to borrow from His Majesty's subjects as much as three millions more than was anticipated at the time when they were called upon to advance it; so that in the course of two or three years we have raised our debt by upwards of twelve million pounds. \n",
      " The most remarkable circumstance, perhaps, of this unprecedented scene of public distress and alarm, is the very great difference which has taken place since its commencement, in the conduct and spirit of different descriptions of people towards each other. The poor man's heart is now overflowing with gratitude; and he who was the first to suffer from this most dreadful calamity, is the last to whom we are disposed to impute any blame for what has occurred. But it is not so much from his own individual misfortunes that he now finds comfort, as because he knows that every man around him is suffering in the like proportion: and that, upon the whole, no class of people have ever been subjected to more cruel oppressions than they.\n",
      "These sentiments may be regarded by many persons as highly dangerous and subversive of all order and government; but the history of human nature is a sufficient answer to their apprehensions. The poor man has suffered till he cannot endure it: the rich man has suffered because he did not suffer sooner. As for the middle classes, they are beginning to find that they have no cause to complain so much as they had supposed; and in reality very little suffering remains.\n",
      "A circumstance of great importance in the present state of Ireland, is the number of persons who are actually unemployed, without having either a regular or sufficient occupation. A gentleman was lately told that there were not less than 800,000 people at this moment in this situation; and he has since been assured by others that the number of them exceeds one million. If this report is true, it will be found upon due inquiry to be the most important subject on which we\n",
      "llama_print_timings:        load time =   15151.57 ms\n",
      "llama_print_timings:      sample time =      75.94 ms /  1024 runs   (    0.07 ms per token, 13484.33 tokens per second)\n",
      "llama_print_timings: prompt eval time =   16269.95 ms /   494 tokens (   32.94 ms per token,    30.36 tokens per second)\n",
      "llama_print_timings:        eval time =  382830.23 ms /  1023 runs   (  374.22 ms per token,     2.67 tokens per second)\n",
      "llama_print_timings:       total time =  399374.15 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# try cpu\n",
    "!./main --color --no-mmap -ngl 0 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74a84502",
   "metadata": {},
   "source": [
    "### 13B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "43038979-9395-4119-94c8-5a8b90198064",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702005725\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 363 tensors from ./models/13B/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  291:             blk.32.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  292:             blk.32.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  293:             blk.32.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  294:        blk.32.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  295:           blk.32.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  296:           blk.32.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  297:             blk.32.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  298:          blk.32.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  299:           blk.32.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  300:             blk.33.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  301:             blk.33.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  302:             blk.33.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  303:        blk.33.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  304:           blk.33.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  305:           blk.33.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  306:             blk.33.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  307:          blk.33.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  308:           blk.33.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  309:             blk.34.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  310:             blk.34.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  311:             blk.34.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  312:        blk.34.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  313:           blk.34.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  314:           blk.34.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  315:             blk.34.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  316:          blk.34.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  317:           blk.34.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  318:             blk.35.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  319:             blk.35.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  320:             blk.35.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  321:        blk.35.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  322:           blk.35.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  323:           blk.35.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  324:             blk.35.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  325:          blk.35.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  326:           blk.35.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  327:             blk.36.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  328:             blk.36.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  329:             blk.36.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  330:        blk.36.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  331:           blk.36.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  332:           blk.36.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  333:             blk.36.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  334:          blk.36.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  335:           blk.36.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  336:             blk.37.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  337:             blk.37.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  338:             blk.37.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  339:        blk.37.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  340:           blk.37.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  341:           blk.37.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  342:             blk.37.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  343:          blk.37.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  344:           blk.37.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  345:             blk.38.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  346:             blk.38.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  347:             blk.38.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  348:        blk.38.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  349:           blk.38.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  350:           blk.38.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  351:             blk.38.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  352:          blk.38.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  353:           blk.38.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  354:             blk.39.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  355:             blk.39.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  356:             blk.39.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  357:        blk.39.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  358:           blk.39.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  359:           blk.39.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  360:             blk.39.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  361:          blk.39.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  362:           blk.39.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: mem required  = 7024.03 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 78.07 MiB\n",
      "llama_new_context_with_model: max tensor size =   128.17 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  7024.03 MiB, ( 7029.73 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   400.03 MiB, ( 7429.77 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    75.02 MiB, ( 7504.78 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentineling the walls of towns against thieves, of whom she had always an ample supply; ferocious legislation, and occasional fires and fields of the guillotine; while it was not until after water had been poured upon her, and many of her priests had been knocked down, by those very fingers which now disdain to touch them, that she called off her doctors and lawyers from their purpose, and set about perfecting her great principle of equal liberty. \n",
      "We have seen the brilliancy that has arisen, we have seen the darkness that has gone down. The evening comes; the lamps are lit in the churches of our country. It is again calm to-night.\n",
      "# – 19 –\n",
      "The Greatness of England\n",
      "Awake! for Morning in the bowl  \n",
      "Of Night hath flung the round Ocean,  \n",
      "And the whiteness of the Day-star  \n",
      "Blazes above it. Awake! arise;  \n",
      "Though our night be long, the morning cometh,  \n",
      "Nevertheless the light is kindled in our mind,  \n",
      "And we have power to shake off sleep.\n",
      "Sooner or later the day must come. As men's eyes get open, they begin to ask how the old order came about, and why it continues. And these questions are being asked by many people, and with greater seriousness than ever before. This is the time for great thinkers and writers to speak, for there will be no other time like it again. We want a new age. Shall we have it?\n",
      "It cannot be said that our ancestors were wholly uncivilised: they built themselves towns and cathedrals, made themselves fine clothes, had music of the choir kind, had books, pictures, jewellery, porcelain – all of which are as beautiful in their way as the art of savages. But there was no real freedom; and there were many things wanting that make life worth having, and among them is this: they thought and wrote a good deal about liberty, and yet had not the spirit to go beyond words, or even far enough to set up a constitution that would be respected by men who have been used for so long to arbitrary power. It may be that our great-grandchildren will laugh at us; but if they are really wise men and women, then let them thank the past and pray that it shall not again come into being in future times.\n",
      "At the present moment we have many people among us who would be called good men and true men and women, and some of these seem to believe that it will never do for us to make any more changes; but I should like to ask them why they believe this. Have they ever considered what life is now like for those who are not born free?\n",
      "The people whose minds are unchanged by all the evidence before their eyes, are as stupid as a cow would be if it could talk, and we know how much sense there is in cows. What can we do about them? Nothing at present; but we shall have to consider this question later on: meanwhile let us leave them alone.\n",
      "It may also happen that men who seem clever and well-intentioned are only trying to make things difficult for others by holding out false hopes of an unchangeable condition, and telling the people that what they want is already in existence among them. Whenever I see this kind of thing, I believe it means the writer has had no practical experience whatever of real life.\n",
      "This is not meant as an offence; but when we look into the lives of others, how can we tell if any particular thing will be helpful to us unless our own minds have been thoroughly disciplined, so that we know what things are necessary or useless?\n",
      "In the first place a man has got to prove himself honest by living up to his word; then he must show himself reliable in small matters before he is entrusted with big ones. These two qualities – honesty and reliability – can only come from thorough discipline of the mind, which means that all the senses are well controlled. It will never do for a man or woman to think they have got everything ready for the future until they have found out whether their eyes are rightly directed. The person who has a steady eye is able to see everything in its true relation: it is only when this condition can be fulfilled that real knowledge becomes possible.\n",
      "I shall now attempt to explain something more about what I mean by the word 'mind.' When we say that mankind must control their minds, we do not mean that they are ever to get rid of all sensation and feeling; for without these things man would become no better than\n",
      "llama_print_timings:        load time =    2790.47 ms\n",
      "llama_print_timings:      sample time =     138.52 ms /  1024 runs   (    0.14 ms per token,  7392.65 tokens per second)\n",
      "llama_print_timings: prompt eval time =    5085.24 ms /   494 tokens (   10.29 ms per token,    97.14 tokens per second)\n",
      "llama_print_timings:        eval time =   92540.01 ms /  1023 runs   (   90.46 ms per token,    11.05 tokens per second)\n",
      "llama_print_timings:       total time =   97937.72 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5de46ea1-0750-477e-9183-d1e6e0cc9e2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702005826\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 363 tensors from ./models/13B/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  291:             blk.32.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  292:             blk.32.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  293:             blk.32.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  294:        blk.32.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  295:           blk.32.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  296:           blk.32.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  297:             blk.32.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  298:          blk.32.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  299:           blk.32.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  300:             blk.33.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  301:             blk.33.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  302:             blk.33.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  303:        blk.33.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  304:           blk.33.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  305:           blk.33.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  306:             blk.33.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  307:          blk.33.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  308:           blk.33.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  309:             blk.34.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  310:             blk.34.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  311:             blk.34.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  312:        blk.34.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  313:           blk.34.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  314:           blk.34.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  315:             blk.34.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  316:          blk.34.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  317:           blk.34.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  318:             blk.35.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  319:             blk.35.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  320:             blk.35.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  321:        blk.35.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  322:           blk.35.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  323:           blk.35.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  324:             blk.35.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  325:          blk.35.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  326:           blk.35.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  327:             blk.36.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  328:             blk.36.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  329:             blk.36.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  330:        blk.36.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  331:           blk.36.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  332:           blk.36.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  333:             blk.36.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  334:          blk.36.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  335:           blk.36.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  336:             blk.37.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  337:             blk.37.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  338:             blk.37.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  339:        blk.37.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  340:           blk.37.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  341:           blk.37.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  342:             blk.37.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  343:          blk.37.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  344:           blk.37.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  345:             blk.38.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  346:             blk.38.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  347:             blk.38.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  348:        blk.38.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  349:           blk.38.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  350:           blk.38.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  351:             blk.38.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  352:          blk.38.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  353:           blk.38.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  354:             blk.39.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  355:             blk.39.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  356:             blk.39.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  357:        blk.39.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  358:           blk.39.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  359:           blk.39.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  360:             blk.39.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  361:          blk.39.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  362:           blk.39.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: mem required  = 7024.03 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 78.07 MiB\n",
      "llama_new_context_with_model: max tensor size =   128.17 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  7024.03 MiB, ( 7029.73 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   400.03 MiB, ( 7429.77 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    75.02 MiB, ( 7504.78 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentencing a youth to have his hands cut off, his tongue torn out with pincers, and his body burned alive, because he had not kneeled down in the rain to do honour to a dirty procession of monks which passed within his view, at a distance of some fifty or sixty yards. It is likely enough that these fine old days of simple ignorance and violence are not yet gone by. But even the shortest memory cannot reach back to any period that does not show something nobil\n",
      "[The manuscript breaks off here; in another copy we have: –]\n",
      "It was the Dauphiness who had said, in the same breath with that of admiring the gems and jewellery of a lady much younger than herself, \"They are all very well; but what do you think of my son? Don't you think him handsome?\" \n",
      "Chapter 36\n",
      "A new world.\n",
      "When Miss Havisham opened her eyes next morning she was amazed to find herself in bed. It is hard to say whether she had slept much or not, but when she woke it was broad day; and the old woman who took charge of the house seemed to be busily engaged about her room. She was sitting up in bed, and the first sound that fell upon her ear was a low voice, saying \"Wheels go round, wheels go round,\" as if it were a lesson being learnt; but it was impossible to understand what these words could mean.\n",
      "Then a harsher voice said, \"What's this? What does the lady say?\"\n",
      "\"It is all right!\" replied the first voice in an authoritative tone.\n",
      "Again came the words, \"Wheels go round, wheels go round,\" and still there was no meaning to be got out of them. Then the harsh voice said sharply, as if it were replying to something:\n",
      "\"It's a trick! It's a trick!\" and thereupon Miss Havisham fell asleep again.\n",
      "When she next opened her eyes, she found herself in the same place, and saw the same old woman sitting by the fire in the kitchen. There was no one else in the house. She soon rose and made the breakfast, while Miss Havisham looked on, wondering what the new day meant to her, but knowing that it brought only trouble and disappointment.\n",
      "She felt very ill at ease, and ashamed of herself; so she turned towards the window, saying that she wished to be left alone for a little time. The old woman took off her shoes and crept on tiptoe out of the room, with her bare feet making no more noise than if they had been covered with thick leather slippers.\n",
      "It was now close upon ten o'clock in the morning, and Miss Havisham still sat at the window, when a cart came rattling up to the door; a man alighted from it with his wagoner's whip over his shoulder, and, opening a gate half-way down the garden, led two large rough dogs towards her. There was a scraping of their paws upon the gravel, a howl of fury as they came near Miss Havisham's window, and she fell back with terror upon her bed; for these were the first dogs that she had seen since she was a child—since Estella left her—and she shuddered and trembled from head to foot.\n",
      "They kept barking and snarling at each other, until the man called them off. He then came into Miss Havisham's room and stood before her, holding out his hand for the money she owed him for bringing Estella home on that awful night more than twenty years ago. He looked much as he had done then; but not quite so young—and less like a sulky bull than he was in those old times. He said to Miss Havisham, \"What's your answer?\"\n",
      "She answered that she would give him the money when his load was gone. She had always intended to keep her word to this poor wagoner and pay him what she owed him for bringing home Estella on that terrible night; but she now felt too ill at ease under his eyes to give him the money in a civil manner, and she had not the courage to dismiss him rudely.\n",
      "\"If you want to be rid of me, Miss Havisham,\" he said, \"I'll go and take the dogs off.\" So he left her and took them out, leaving the gate open behind him. As they went growling down the garden like a pair of large vicious bull-dogs, they rattled\n",
      "llama_print_timings:        load time =    2529.50 ms\n",
      "llama_print_timings:      sample time =     171.42 ms /  1024 runs   (    0.17 ms per token,  5973.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =    5091.17 ms /   494 tokens (   10.31 ms per token,    97.03 tokens per second)\n",
      "llama_print_timings:        eval time =   92059.01 ms /  1023 runs   (   89.99 ms per token,    11.11 tokens per second)\n",
      "llama_print_timings:       total time =   97519.84 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "763256c8-af9e-493e-b52d-4c161814bd72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702005927\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 363 tensors from ./models/13B/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  291:             blk.32.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  292:             blk.32.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  293:             blk.32.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  294:        blk.32.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  295:           blk.32.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  296:           blk.32.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  297:             blk.32.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  298:          blk.32.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  299:           blk.32.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  300:             blk.33.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  301:             blk.33.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  302:             blk.33.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  303:        blk.33.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  304:           blk.33.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  305:           blk.33.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  306:             blk.33.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  307:          blk.33.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  308:           blk.33.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  309:             blk.34.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  310:             blk.34.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  311:             blk.34.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  312:        blk.34.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  313:           blk.34.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  314:           blk.34.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  315:             blk.34.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  316:          blk.34.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  317:           blk.34.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  318:             blk.35.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  319:             blk.35.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  320:             blk.35.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  321:        blk.35.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  322:           blk.35.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  323:           blk.35.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  324:             blk.35.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  325:          blk.35.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  326:           blk.35.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  327:             blk.36.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  328:             blk.36.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  329:             blk.36.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  330:        blk.36.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  331:           blk.36.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  332:           blk.36.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  333:             blk.36.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  334:          blk.36.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  335:           blk.36.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  336:             blk.37.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  337:             blk.37.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  338:             blk.37.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  339:        blk.37.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  340:           blk.37.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  341:           blk.37.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  342:             blk.37.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  343:          blk.37.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  344:           blk.37.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  345:             blk.38.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  346:             blk.38.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  347:             blk.38.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  348:        blk.38.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  349:           blk.38.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  350:           blk.38.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  351:             blk.38.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  352:          blk.38.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  353:           blk.38.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  354:             blk.39.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  355:             blk.39.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  356:             blk.39.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  357:        blk.39.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  358:           blk.39.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  359:           blk.39.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  360:             blk.39.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  361:          blk.39.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  362:           blk.39.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = models\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 2048\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 2048\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 2048\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = models\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: mem required  = 7024.03 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 78.07 MiB\n",
      "llama_new_context_with_model: max tensor size =   128.17 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  7024.03 MiB, ( 7029.73 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   400.03 MiB, ( 7429.77 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    75.02 MiB, ( 7504.78 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentineling the streets of Paris, and burning people in the public square who held languages and opinions in high disfavour. She had also some more quiet and sedative systems than the rack and fagot, and one of them was called the Law of Hostages. \n",
      "On the fifteenth of July there broke out a riot in the Rue Saint Honore, which that day chanced to be deserted; a putrid mass of rotten fish (which had contrived to espy a stagnant pool among the cobble-stones) engendering this general exodus. At the end of two hours' fighting, three hundred and eighty-five people were arrested; whereof twenty-six were shipped for America, and the remainder consigned to the prisons of France or Spain: and in the meantime the loungers of the quarter arméd themselves with pikes, bludgeons, and every variety of rude weapon, and laid siege to the Abbaye. \n",
      "The prison stood a little way from the Rue Saint Honore. Its walls were twenty feet high; nothing could be seen from without; a wicket alone gave admission; the place was tripled by an outer court, a middle court, and an inner court; each court having its separate door, and being accessible only to the governor of the prison, or to persons under his authority: so that to gain admittance to any prisoner was no easy task. The prisoners, however, had found means to secrete arms within the Abbaye. They opened their cells upon those of their comrades, who were also arrested; and these again were communicated with other prisons. Thus they contrived a subterranean passage into the dungeon where Milady was confined: this dungeon was dark as well as horrible; it was composed of three cells, one over another, with only an opening between them in width of two feet and a half, and from that opening to the outside world there was but a single flight of stairs, winding around a square tower. In the lowest cell there was a bed of filth; some bones lay scattered about, mouldering away upon a heap of putrid rags; in this loathsome abode there remained only a small quantity of stale water and a few morsels of rotten food, which had served for four days for the nourishment of three prisoners.\n",
      "\n",
      "One day one of these prisoners was seized with an epileptic attack, which lasted until he became nearly dead; another prisoner fell down in a fit. The third and fourth prisoners, whose cells were above that of the first, had their feet broken by stones thrown at them by the epileptic, whom they could not reach from below because of his suffering and unconsciousness.\n",
      "\n",
      "Thereupon the governor ordered the three most violent to be put in irons, and the fourth into a solitary cell on account of his infirmity. There only remained one person out of four with his hands at liberty.\n",
      "\n",
      "That day was the third since the confinement of Milady. She had been warned of her fate by an old prisoner, who informed her that her turn to die was about to come; and the unhappy woman did not doubt it for a moment.\n",
      "\n",
      "The four prisoners who remained free from irons resolved to attempt an escape, so as to save at least one life. It will be seen what means they took to this end: in order that there might remain some account of it hereafter, we shall now give the reader a faithful description.\n",
      "\n",
      "The two cells below were darker than the others, because their windows looked only on the outer wall, without any air or light from within; and no communication could take place between them except by means of an iron trapdoor in the floor which gave access to a sort of well in the depths of the tower.\n",
      "\n",
      "The two prisoners who were left free went down into this subterraneous prison, where they found the epileptic stretched out senseless on his bed, and the fourth prisoner also lying there, but alive. He was a man of great strength, though weakened by a fever, which had kept him confined to his bed since his arrival at Vincennes; it was this man who had been left free from irons.\n",
      "\n",
      "“There are two ways out,” he said: “there is the well, and there is a small window on that side which opens into a sort of shed. But the first has only one drawbridge, while the other has two. Besides, the two iron bars of which it was composed, one over the other, are not taken down often; but\n",
      "llama_print_timings:        load time =    2469.65 ms\n",
      "llama_print_timings:      sample time =     111.17 ms /  1024 runs   (    0.11 ms per token,  9211.45 tokens per second)\n",
      "llama_print_timings: prompt eval time =    5099.38 ms /   494 tokens (   10.32 ms per token,    96.87 tokens per second)\n",
      "llama_print_timings:        eval time =   92195.59 ms /  1023 runs   (   90.12 ms per token,    11.10 tokens per second)\n",
      "llama_print_timings:       total time =   97573.28 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
